{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "### Set up proxy to access hugging face library\n",
    "import os\n",
    "\n",
    "os.environ[\"HTTP_PROXY\"] = \"http://proxy-dku.oit.duke.edu:3128\"\n",
    "os.environ[\"HTTPS_PROXY\"] = \"http://proxy-dku.oit.duke.edu:3128\"\n",
    "\n",
    "### import packages\n",
    "from transformers import AutoProcessor,WavLMModel, AutoModel\n",
    "import torch\n",
    "from datasets import load_dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\torch\\_utils.py:776: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
      "  return self.fget.__get__(instance, owner)()\n"
     ]
    }
   ],
   "source": [
    "model = AutoModel.from_pretrained(\"microsoft/wavlm-large\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "### randomly generate audio, here the batch size is 2\n",
    "audio = torch.randn(2,16000*20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wav2Vec2BaseModelOutput(last_hidden_state=tensor([[[-0.3784, -0.0016, -0.4596,  ..., -0.1236, -0.1294, -0.1358],\n",
      "         [-0.3657,  0.0544, -0.5358,  ..., -0.1036, -0.1933, -0.1362],\n",
      "         [-0.3564,  0.0672, -0.4977,  ..., -0.1213, -0.1833, -0.0985],\n",
      "         ...,\n",
      "         [-0.3688,  0.0134, -0.4067,  ...,  0.0032, -0.2446, -0.1936],\n",
      "         [-0.3828,  0.0139, -0.3928,  ..., -0.0427, -0.2463, -0.1160],\n",
      "         [-0.3299,  0.0468, -0.3189,  ..., -0.0420, -0.2244, -0.1322]],\n",
      "\n",
      "        [[-0.3626,  0.0260, -0.4058,  ..., -0.1254, -0.1057, -0.0410],\n",
      "         [-0.3140,  0.0339, -0.4643,  ..., -0.1246, -0.1213, -0.1267],\n",
      "         [-0.3249,  0.0698, -0.5137,  ..., -0.0857, -0.1562, -0.1067],\n",
      "         ...,\n",
      "         [-0.2675,  0.0508, -0.4767,  ..., -0.0773, -0.2989, -0.1043],\n",
      "         [-0.3510,  0.0958, -0.4259,  ..., -0.0913, -0.3067, -0.1233],\n",
      "         [-0.3109,  0.1279, -0.3786,  ..., -0.0266, -0.2726, -0.1186]]],\n",
      "       grad_fn=<NativeLayerNormBackward0>), extract_features=tensor([[[ 0.0694,  0.1974,  0.0082,  ..., -0.1224,  0.0257,  0.3124],\n",
      "         [-0.1335, -0.3475,  0.0657,  ..., -0.1357,  0.0222,  0.2867],\n",
      "         [-0.1729, -0.2058, -0.0309,  ...,  0.2733,  0.0329,  0.0088],\n",
      "         ...,\n",
      "         [ 0.1791, -0.1388, -0.0036,  ..., -0.2900,  0.0261,  0.3602],\n",
      "         [ 0.0990, -0.5235, -0.0477,  ..., -0.0944,  0.0292,  0.4512],\n",
      "         [ 0.1680,  0.2021, -0.0670,  ...,  0.0270,  0.0344,  0.0505]],\n",
      "\n",
      "        [[ 0.1931, -0.0623,  0.0471,  ..., -0.1308,  0.0313,  0.4784],\n",
      "         [ 0.0632,  0.2821,  0.0670,  ...,  0.1074,  0.0225,  0.6245],\n",
      "         [-0.1875,  0.8073,  0.0111,  ..., -0.0500,  0.0308,  0.4843],\n",
      "         ...,\n",
      "         [ 0.0651,  0.4127,  0.0385,  ...,  0.1005,  0.0334,  0.5872],\n",
      "         [-0.0342,  0.3079,  0.0931,  ..., -0.3852,  0.0335,  0.3520],\n",
      "         [ 0.2589,  0.5207,  0.0998,  ..., -0.0982,  0.0192,  0.5249]]],\n",
      "       grad_fn=<NativeLayerNormBackward0>), hidden_states=(tensor([[[ 2.5832,  2.3095,  2.6157,  ...,  1.2703,  1.3607,  1.5121],\n",
      "         [ 2.2479,  0.8667,  3.7746,  ...,  1.2335,  1.0596,  0.5361],\n",
      "         [ 2.7122,  1.7768,  3.1964,  ...,  0.0497,  2.1906,  1.7229],\n",
      "         ...,\n",
      "         [ 0.7801,  2.1361,  1.5152,  ...,  1.4701,  0.6806,  0.8111],\n",
      "         [ 1.3864,  0.7769,  2.3457,  ..., -0.1498,  0.9559,  1.1841],\n",
      "         [ 1.4372,  0.8055,  2.5123,  ..., -0.4810,  0.3823,  0.8676]],\n",
      "\n",
      "        [[ 0.9399,  1.8027,  1.4260,  ...,  1.2018,  1.5311,  1.5364],\n",
      "         [ 2.0938,  2.0748,  2.9324,  ...,  0.8729,  0.6941,  1.1540],\n",
      "         [ 2.7133,  1.9389,  2.3062,  ...,  0.4114,  0.5722,  0.7260],\n",
      "         ...,\n",
      "         [ 1.9445,  2.1886,  1.1626,  ...,  0.3769,  0.8379,  0.5314],\n",
      "         [ 1.0977,  3.2504,  3.3375,  ...,  1.0675,  0.1689,  1.5333],\n",
      "         [ 1.1329,  2.7144,  3.3219,  ...,  0.9688,  0.9916, -0.3561]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-1.4538, -1.9089, -0.4221,  ...,  1.1472,  2.4518,  2.4685],\n",
      "         [-0.7814, -1.0265, -0.0712,  ...,  2.5290,  2.1512,  1.0039],\n",
      "         [ 0.6497, -0.3117,  0.2261,  ...,  0.9673,  1.1077,  0.5515],\n",
      "         ...,\n",
      "         [-0.7832, -0.6244, -1.1158,  ...,  2.3311,  0.5601,  1.5038],\n",
      "         [-0.2852, -1.2420,  0.0226,  ...,  0.1353,  0.8233,  1.2741],\n",
      "         [ 0.1128, -2.7068, -0.0945,  ...,  0.7052,  0.7297,  1.2979]],\n",
      "\n",
      "        [[-2.0940, -2.8602, -0.9759,  ...,  0.5180,  1.7632,  2.3766],\n",
      "         [-0.8815, -1.4642, -0.8059,  ...,  2.1524,  0.6492,  1.2958],\n",
      "         [ 1.1430, -0.4135, -0.2546,  ...,  1.8684,  0.3241,  0.2825],\n",
      "         ...,\n",
      "         [-0.0081, -1.0239, -1.4852,  ...,  1.0881,  0.8191,  0.1423],\n",
      "         [-0.0243,  0.5468,  0.9499,  ...,  2.3727,  0.3347,  1.6758],\n",
      "         [-0.7661, -0.2871,  0.3955,  ...,  1.0447,  1.4759, -0.4158]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-3.2866, -0.5638,  0.7971,  ...,  0.7969,  1.4246,  3.8486],\n",
      "         [-0.5087,  0.3422, -0.1703,  ...,  2.5877,  0.4749,  1.5693],\n",
      "         [ 0.0998,  1.8258,  0.5844,  ...,  1.9480, -1.6808,  1.2125],\n",
      "         ...,\n",
      "         [-0.6844,  0.6189, -1.1343,  ...,  2.5160, -1.7739,  3.3587],\n",
      "         [ 0.1236, -0.4431,  0.4675,  ...,  0.6549, -1.0919,  1.9639],\n",
      "         [-0.8467, -0.2510, -0.6992,  ...,  1.4086, -0.0504,  1.9036]],\n",
      "\n",
      "        [[-3.7254, -0.7142,  0.3487,  ...,  0.6744, -0.0254,  3.5231],\n",
      "         [-0.5923, -0.1572, -1.0492,  ...,  2.4858, -1.5084,  2.5579],\n",
      "         [ 1.1035,  1.0388,  0.4869,  ...,  2.7742, -1.6233,  1.8332],\n",
      "         ...,\n",
      "         [ 0.6368,  0.3653, -0.7017,  ...,  1.8730, -1.2332,  0.8730],\n",
      "         [ 1.9472,  1.1838,  1.9916,  ...,  2.9805, -0.0896,  3.5473],\n",
      "         [-1.6565,  1.0454, -0.4754,  ...,  2.3613,  0.3664,  0.8326]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-1.8015, -0.1713,  1.4086,  ...,  2.9625,  0.7817,  2.7425],\n",
      "         [ 1.2663,  0.4735, -1.1872,  ...,  4.1243, -0.0966, -0.1910],\n",
      "         [ 1.5748,  3.0514,  0.4159,  ...,  3.8952, -2.9707, -0.0980],\n",
      "         ...,\n",
      "         [-0.2919,  0.6015, -1.5224,  ...,  5.0909, -2.7935,  2.1820],\n",
      "         [ 1.0865,  0.3331,  0.9052,  ...,  3.2993, -2.5975,  1.6107],\n",
      "         [-0.3244, -0.2044, -0.7748,  ...,  3.3780, -0.2643,  1.0596]],\n",
      "\n",
      "        [[-2.6848,  0.3916,  1.0712,  ...,  2.7562,  0.0327,  1.7178],\n",
      "         [ 0.9414,  0.4369, -0.3313,  ...,  5.0037, -2.3359,  1.6959],\n",
      "         [ 2.4635,  2.0336,  0.5474,  ...,  4.7696, -2.0939,  0.6560],\n",
      "         ...,\n",
      "         [ 0.6648,  0.0564, -1.1100,  ...,  3.9495, -1.9671,  0.2202],\n",
      "         [ 2.9826,  1.0715,  1.9335,  ...,  5.5361, -0.8824,  2.9417],\n",
      "         [-0.7741,  1.3373, -0.6385,  ...,  3.9832, -0.7745, -0.3049]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-3.7889, -4.6424,  2.5789,  ...,  1.4096, -0.5148,  3.9912],\n",
      "         [-0.8790, -2.9914, -0.2991,  ...,  4.4457, -3.3632,  0.6769],\n",
      "         [-0.8938, -1.7627,  1.4255,  ...,  4.7672, -6.1415,  0.9900],\n",
      "         ...,\n",
      "         [-2.3404, -4.1165,  0.3970,  ...,  5.4390, -6.2116,  1.9408],\n",
      "         [-0.4439, -4.5755,  2.5026,  ...,  3.7253, -6.2116,  1.5481],\n",
      "         [-0.8513, -3.9582,  2.2885,  ...,  3.2276, -1.8562,  0.3725]],\n",
      "\n",
      "        [[-3.9977, -4.3770,  2.9161,  ...,  1.1223,  0.1213,  3.3353],\n",
      "         [-0.5168, -4.5964,  0.5004,  ...,  4.3755, -5.1351,  3.3831],\n",
      "         [ 0.5057, -2.1192,  2.2939,  ...,  5.0288, -5.1392,  1.8512],\n",
      "         ...,\n",
      "         [-1.3305, -4.7215, -0.3614,  ...,  4.8103, -6.0164,  0.2843],\n",
      "         [ 0.9006, -2.8284,  3.0894,  ...,  5.3215, -5.0910,  2.1659],\n",
      "         [-1.3329, -2.0825,  1.7197,  ...,  4.0881, -3.5878, -0.7625]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-3.2168, -4.5301, -0.2991,  ...,  0.0243, -2.0091,  6.0387],\n",
      "         [ 0.1423, -4.7344, -4.1575,  ...,  5.2165, -5.7299,  2.9874],\n",
      "         [-1.5807, -4.0228, -1.8703,  ...,  5.6562, -8.5412,  2.7462],\n",
      "         ...,\n",
      "         [-1.6682, -5.6542, -2.2850,  ...,  8.5556, -7.0563,  3.9425],\n",
      "         [ 0.5949, -6.1435, -0.3066,  ...,  5.6648, -7.8116,  4.0445],\n",
      "         [-0.6513, -3.7802, -2.1725,  ...,  3.3112, -1.9827,  2.8267]],\n",
      "\n",
      "        [[-3.2935, -3.9793,  0.7003,  ...,  0.9269, -0.8686,  4.8861],\n",
      "         [ 0.3662, -5.9020, -1.9736,  ...,  5.8680, -6.7401,  5.3625],\n",
      "         [ 0.6573, -3.2556, -1.4972,  ...,  6.7647, -6.9022,  3.7263],\n",
      "         ...,\n",
      "         [-1.3256, -6.2158, -3.7424,  ...,  5.7412, -6.7229,  2.1477],\n",
      "         [ 1.7084, -5.0133, -0.5335,  ...,  6.6393, -5.2667,  3.1396],\n",
      "         [-1.0974, -2.4543, -1.9087,  ...,  4.5073, -4.2772,  1.4380]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-4.3127, -7.2441, -1.5686,  ...,  0.3030,  0.7115,  4.4032],\n",
      "         [ 0.5888, -4.8285, -4.5704,  ...,  5.6111, -3.9972, -0.9062],\n",
      "         [-0.2834, -3.4264, -2.4909,  ...,  7.0879, -6.7975,  0.1490],\n",
      "         ...,\n",
      "         [-1.3335, -6.3011, -3.0055,  ...,  8.7093, -5.0121,  3.9609],\n",
      "         [ 0.4933, -7.7236, -1.6702,  ...,  5.2025, -5.3248,  3.1282],\n",
      "         [-2.0030, -6.8500, -2.4013,  ...,  3.0415,  1.2439,  1.0427]],\n",
      "\n",
      "        [[-3.8162, -7.2687, -0.6046,  ...,  0.1349,  2.9476,  2.6038],\n",
      "         [ 0.9348, -7.3436, -2.8676,  ...,  5.9999, -4.1626,  1.9606],\n",
      "         [ 0.9394, -4.5198, -2.6558,  ...,  8.0986, -4.7001,  1.3186],\n",
      "         ...,\n",
      "         [-1.8152, -7.3900, -5.3329,  ...,  6.4029, -4.8028,  0.2126],\n",
      "         [ 1.7612, -5.8043, -1.3718,  ...,  8.0411, -3.5981,  1.9399],\n",
      "         [-2.2706, -4.8912, -2.5796,  ...,  4.5406, -1.0726,  0.1600]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-2.0310, -6.1243, -0.9684,  ..., -1.2613,  0.7938,  7.4965],\n",
      "         [ 2.0469, -2.5013, -2.7396,  ...,  4.9042, -5.2527,  4.2285],\n",
      "         [ 2.4034, -0.3065, -1.4536,  ...,  6.7925, -7.8916,  5.2829],\n",
      "         ...,\n",
      "         [ 0.6143, -3.1822, -1.1447,  ...,  9.6951, -6.1375,  9.0336],\n",
      "         [ 4.0326, -5.0186, -0.1034,  ...,  6.6681, -5.9960,  8.8627],\n",
      "         [ 2.0462, -4.0274, -0.4079,  ...,  1.4867,  0.3589,  3.5640]],\n",
      "\n",
      "        [[-1.3904, -4.4685,  0.9274,  ..., -0.8919,  3.5529,  5.1667],\n",
      "         [ 3.0484, -4.8336, -0.3046,  ...,  5.2215, -5.0914,  6.1275],\n",
      "         [ 3.0400, -1.6378, -0.7343,  ...,  7.4866, -6.0226,  6.0521],\n",
      "         ...,\n",
      "         [-1.1129, -4.3609, -6.1659,  ...,  6.1892, -5.4050,  5.5611],\n",
      "         [ 3.7453, -1.9803, -0.0937,  ...,  8.3746, -5.0985,  7.9727],\n",
      "         [ 1.7789, -2.2790,  0.3645,  ...,  3.7427, -2.0455,  2.8063]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-5.1928, -8.5218, -0.2524,  ..., -7.3372,  3.8908,  7.4491],\n",
      "         [-1.8704, -6.6404, -1.8601,  ...,  4.8595, -1.8242,  5.4000],\n",
      "         [-0.3637, -4.8123, -1.1797,  ...,  6.8273, -3.3905,  5.2521],\n",
      "         ...,\n",
      "         [-0.5133, -6.3028, -0.1251,  ...,  8.2777, -3.0933,  8.1210],\n",
      "         [ 3.5292, -8.3834,  0.4325,  ...,  4.7366, -4.6960,  9.3680],\n",
      "         [ 0.1388, -6.4510, -0.1595,  ..., -2.8097,  2.2298,  4.4087]],\n",
      "\n",
      "        [[-5.3686, -6.3874,  1.6207,  ..., -7.1216,  5.0952,  5.7016],\n",
      "         [-0.5715, -8.2866, -1.1718,  ...,  3.6844, -1.6641,  6.8556],\n",
      "         [-0.2072, -5.0359,  0.3495,  ...,  6.7164, -1.1336,  6.1356],\n",
      "         ...,\n",
      "         [-2.8428, -8.2032, -6.0597,  ...,  3.3986, -4.0155,  6.0208],\n",
      "         [ 1.9744, -6.8166,  0.0178,  ...,  7.8710, -3.9089,  8.8338],\n",
      "         [-0.8432, -5.3376,  0.8618,  ...,  0.0540, -0.2126,  4.5493]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-5.1641e+00, -7.2749e+00, -5.5045e-01,  ..., -2.4389e+00,\n",
      "           3.1695e+00,  3.9876e+00],\n",
      "         [-6.0626e-01, -5.9114e+00, -3.1836e+00,  ...,  1.0634e+01,\n",
      "          -1.6847e+00,  1.0528e+00],\n",
      "         [ 4.2500e-01, -5.3837e+00, -3.6642e+00,  ...,  1.1501e+01,\n",
      "          -1.8764e+00,  1.9895e+00],\n",
      "         ...,\n",
      "         [-3.1434e-02, -8.7176e+00, -3.3710e-03,  ...,  1.3087e+01,\n",
      "          -3.4253e+00,  4.8715e+00],\n",
      "         [ 4.5263e+00, -1.0501e+01,  1.0560e+00,  ...,  9.5532e+00,\n",
      "          -4.5302e+00,  5.5215e+00],\n",
      "         [ 5.8426e-01, -6.1160e+00, -1.3838e-01,  ...,  1.4852e+00,\n",
      "           2.9317e+00,  1.2665e+00]],\n",
      "\n",
      "        [[-5.9789e+00, -4.9580e+00,  5.4011e-01,  ..., -2.2376e+00,\n",
      "           4.5587e+00,  1.6028e+00],\n",
      "         [ 2.0296e-01, -7.7841e+00, -3.4489e+00,  ...,  8.9343e+00,\n",
      "          -9.9046e-01,  1.6638e+00],\n",
      "         [ 6.8392e-01, -5.4254e+00, -1.2214e+00,  ...,  1.1625e+01,\n",
      "           1.9283e-01,  1.7703e+00],\n",
      "         ...,\n",
      "         [-6.7120e-01, -1.0088e+01, -5.1578e+00,  ...,  7.1782e+00,\n",
      "          -2.4958e+00,  3.9232e+00],\n",
      "         [ 3.7728e+00, -9.0534e+00,  3.7993e-01,  ...,  1.1720e+01,\n",
      "          -3.3615e+00,  6.2055e+00],\n",
      "         [-2.6480e-01, -5.3432e+00,  3.8542e-01,  ...,  3.7520e+00,\n",
      "           3.5390e-01,  1.2120e+00]]], grad_fn=<AddBackward0>), tensor([[[-4.5481, -3.7272, -1.5194,  ..., -7.2906,  6.7395,  3.7328],\n",
      "         [ 2.4034, -3.9807, -4.0481,  ...,  7.1627,  1.3179,  0.0906],\n",
      "         [ 3.3114, -2.5079, -4.6276,  ...,  8.9916,  1.1398,  1.0719],\n",
      "         ...,\n",
      "         [ 2.0403, -3.8943, -2.8296,  ...,  9.5940, -0.8494,  3.0648],\n",
      "         [ 7.6062, -5.7858,  0.0291,  ...,  7.0656, -0.6319,  4.1156],\n",
      "         [ 4.0052, -2.8711, -0.8811,  ..., -3.0091,  5.7380,  0.2847]],\n",
      "\n",
      "        [[-5.0735, -1.8326, -0.4066,  ..., -6.6653,  6.8906,  1.3964],\n",
      "         [ 3.0967, -7.2533, -5.2093,  ...,  6.3616,  1.2398,  1.1402],\n",
      "         [ 4.2716, -4.3743, -2.4726,  ...,  9.5612,  2.2303,  0.8245],\n",
      "         ...,\n",
      "         [ 2.5278, -6.3347, -4.5313,  ...,  3.4757,  0.3903,  1.9506],\n",
      "         [ 6.6943, -5.7221,  0.6482,  ...,  8.2064,  0.1142,  4.4419],\n",
      "         [ 2.9508, -2.5731, -1.2054,  ..., -0.9003,  2.8499,  0.1909]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ -7.7534,  -9.5820,  -1.8453,  ...,  -5.2661,   3.8130,   4.4245],\n",
      "         [  1.9480,  -7.8399,  -4.1854,  ...,   6.7916,  -4.2440,   2.1291],\n",
      "         [  2.5024,  -6.0890,  -5.3837,  ...,   8.3006,  -3.9156,   2.8379],\n",
      "         ...,\n",
      "         [  0.6284,  -7.5175,  -4.4747,  ...,   9.1561,  -5.3657,   4.1534],\n",
      "         [  6.0377, -10.3544,  -1.2123,  ...,   7.7063,  -3.8946,   4.9583],\n",
      "         [  1.1070, -10.9911,  -2.3295,  ...,  -0.8858,   2.7749,   2.2001]],\n",
      "\n",
      "        [[ -7.9560,  -7.1299,  -2.2146,  ...,  -5.3721,   4.4729,   3.3762],\n",
      "         [  0.9468,  -9.5419,  -5.8155,  ...,   6.1604,  -2.9638,   3.4745],\n",
      "         [  3.9090,  -6.3352,  -3.3282,  ...,   9.6084,  -2.1006,   2.5991],\n",
      "         ...,\n",
      "         [  2.7761,  -9.4854,  -5.5741,  ...,   4.0380,  -3.0378,   3.7045],\n",
      "         [  5.5729,  -9.5257,  -0.5899,  ...,   8.1365,  -2.7970,   5.3326],\n",
      "         [ -0.0584,  -9.9274,  -2.9495,  ...,   0.2868,   0.3740,   2.3505]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ -5.6669,  -9.5614,  -7.8126,  ...,  -2.1936,   5.4304,   4.2150],\n",
      "         [  4.8339,  -7.4565, -10.5863,  ...,   8.5369,  -5.8068,   1.0809],\n",
      "         [  4.1603,  -5.3965, -11.3821,  ...,  10.4734,  -4.6810,   2.8307],\n",
      "         ...,\n",
      "         [  5.3661,  -6.3633,  -8.5068,  ...,  11.0596,  -3.5158,   3.9653],\n",
      "         [ 10.0264,  -9.5697,  -5.2936,  ...,   9.4011,  -3.4736,   6.3333],\n",
      "         [  3.8927,  -8.2743,  -7.5015,  ...,   3.8578,   2.2231,   5.0063]],\n",
      "\n",
      "        [[ -5.4669,  -7.2277,  -8.0742,  ...,  -2.1085,   6.1198,   4.2874],\n",
      "         [  3.8794,  -8.6627, -11.2174,  ...,   7.2203,  -3.3207,   3.0059],\n",
      "         [  5.6268,  -4.6062,  -9.1198,  ...,  12.0403,  -1.8422,   1.9792],\n",
      "         ...,\n",
      "         [  7.4171,  -8.4954, -10.4181,  ...,   7.8083,  -2.7751,   2.4834],\n",
      "         [  8.8986,  -6.9786,  -2.4963,  ...,  10.0067,  -3.0869,   4.7751],\n",
      "         [  2.6599,  -6.6660,  -7.5937,  ...,   4.8837,   0.3078,   5.3373]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ -3.3647,  -9.5923,  -6.2805,  ...,  -4.4410,   5.4157,   4.7165],\n",
      "         [  5.9729,  -8.0179,  -6.0711,  ...,   6.0532,  -7.0630,  -0.6302],\n",
      "         [  5.8601,  -4.1349,  -6.3831,  ...,   6.5163,  -6.5993,   2.6083],\n",
      "         ...,\n",
      "         [  6.4013,  -6.4625,  -4.4920,  ...,   4.3490,  -7.3167,   5.7028],\n",
      "         [ 11.5895,  -9.6605,  -1.5346,  ...,   3.5620,  -7.3743,   8.1460],\n",
      "         [  5.6924,  -7.1756,  -6.9845,  ...,  -1.3575,   2.7399,   7.0222]],\n",
      "\n",
      "        [[ -1.9925,  -7.6759,  -6.2574,  ...,  -4.7947,   5.8974,   6.3241],\n",
      "         [  5.9432, -10.5180,  -6.5301,  ...,   4.4052,  -3.3191,   3.4988],\n",
      "         [  7.6343,  -4.2274,  -3.2312,  ...,   7.0263,  -3.8557,   2.9761],\n",
      "         ...,\n",
      "         [  9.0484,  -8.7006,  -5.6812,  ...,   2.7579,  -6.4231,   4.6575],\n",
      "         [  9.6716,  -6.7665,   2.3438,  ...,   3.3894,  -5.5443,   6.3261],\n",
      "         [  4.3089,  -5.4259,  -6.7000,  ...,  -0.3218,   0.7594,   6.8689]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ -3.0593, -10.8994,  -5.2387,  ...,  -3.4778,   8.6012,   2.7718],\n",
      "         [  3.9026,  -9.5848,  -7.2239,  ...,   6.2424,  -3.7052,  -1.4218],\n",
      "         [  3.7346,  -4.7470,  -8.7073,  ...,   7.5404,  -1.9735,   0.4949],\n",
      "         ...,\n",
      "         [  4.1956,  -4.7710,  -6.5292,  ...,   7.6216,  -6.6702,   1.3987],\n",
      "         [  8.8110,  -8.2023,  -4.7878,  ...,   8.0141,  -6.6833,   4.5393],\n",
      "         [  4.7369,  -7.8951,  -7.0463,  ...,   3.0348,   4.0567,   3.3424]],\n",
      "\n",
      "        [[ -0.8703, -10.0303,  -5.3623,  ...,  -3.5718,   9.8563,   4.6107],\n",
      "         [  5.3595, -12.0732,  -9.0401,  ...,   6.0461,   1.3009,   3.1235],\n",
      "         [  4.7206,  -5.1194,  -7.3810,  ...,   9.0345,   1.3533,   0.0518],\n",
      "         ...,\n",
      "         [  7.7519,  -7.8456,  -6.5210,  ...,   5.5848,  -5.7737,  -0.3523],\n",
      "         [  8.4558,  -6.5054,   0.5299,  ...,   6.3429,  -4.9316,   2.5348],\n",
      "         [  4.5644,  -6.0916,  -7.4190,  ...,   4.1889,   2.0724,   3.1602]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ -6.7038,  -9.7254,  -4.6827,  ...,  -5.9198,  11.1767,   1.0137],\n",
      "         [  0.0778, -10.6255,  -7.2802,  ...,   3.4598,  -4.5299,  -2.4347],\n",
      "         [  0.2060,  -6.9763,  -9.2466,  ...,   4.6203,  -3.8273,  -0.4858],\n",
      "         ...,\n",
      "         [ -1.9173,  -5.1264,  -7.1800,  ...,   1.9826,  -6.5452,   1.8446],\n",
      "         [  2.5730,  -7.7585,  -6.3634,  ...,   3.0434,  -5.7167,   4.3166],\n",
      "         [ -1.6536,  -5.4265,  -7.5338,  ...,  -2.4993,   6.4314,   2.8667]],\n",
      "\n",
      "        [[ -4.5843,  -9.0737,  -3.9705,  ...,  -5.4320,  12.0174,   3.4973],\n",
      "         [  2.3263, -12.9620,  -7.8938,  ...,   3.2118,   1.2692,   1.8069],\n",
      "         [  1.1552,  -5.8073,  -7.6062,  ...,   6.7827,  -0.1852,  -1.5627],\n",
      "         ...,\n",
      "         [  1.7881,  -8.1735,  -8.1075,  ...,   0.5158,  -5.7046,   0.0778],\n",
      "         [  2.5025,  -7.0849,  -0.2430,  ...,   0.2956,  -3.6050,   2.9802],\n",
      "         [ -2.1821,  -4.2527,  -8.1176,  ...,  -1.2758,   4.5641,   4.0157]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ -7.1492, -10.1030,  -3.1302,  ...,  -7.0241,  11.0864,   2.2465],\n",
      "         [ -0.6842, -11.8927,  -6.7822,  ...,   3.4938,  -5.0450,  -0.6681],\n",
      "         [ -1.9490,  -8.9938,  -9.3265,  ...,   5.4674,  -4.2782,   0.7889],\n",
      "         ...,\n",
      "         [ -1.8545,  -4.9869,  -5.8075,  ...,   1.8067,  -9.7206,   1.8628],\n",
      "         [  2.9356,  -7.2850,  -4.0184,  ...,   2.2457,  -9.1046,   4.1624],\n",
      "         [ -2.3104,  -5.6470,  -4.5758,  ...,  -3.1375,   3.4343,   2.6663]],\n",
      "\n",
      "        [[ -5.0223,  -9.2541,  -2.0241,  ...,  -6.2171,  11.7836,   4.6662],\n",
      "         [  0.4243, -13.8122,  -8.2435,  ...,   2.1406,   0.3020,   3.8043],\n",
      "         [ -0.6230,  -6.6807,  -7.3225,  ...,   7.2680,  -1.2458,  -0.3428],\n",
      "         ...,\n",
      "         [  2.2332,  -8.2312,  -7.8882,  ...,   0.8949,  -8.4081,  -1.3458],\n",
      "         [  1.3687,  -6.8999,   0.9027,  ...,   0.2286,  -7.7543,   1.4276],\n",
      "         [ -2.9572,  -3.8210,  -4.8994,  ...,  -1.3551,   0.9791,   3.4624]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ -2.9757,  -9.3807,  -1.3803,  ..., -13.3672,   8.4005,   2.1327],\n",
      "         [  2.9286, -13.2012,  -4.8509,  ...,  -1.9734,  -7.8333,  -1.0886],\n",
      "         [  0.2451, -10.3461,  -9.0161,  ...,   0.1151,  -6.5507,   0.0552],\n",
      "         ...,\n",
      "         [ -0.2626,  -4.7511,  -4.4465,  ...,  -0.3164, -12.5235,   1.6098],\n",
      "         [  4.6500,  -6.6022,  -3.2037,  ...,  -0.9347, -11.9946,   3.7927],\n",
      "         [  1.4641,  -4.2628,  -5.1048,  ...,  -5.8311,   2.1327,   2.7720]],\n",
      "\n",
      "        [[ -1.3367,  -8.2347,  -0.3948,  ..., -11.5804,   9.4482,   4.7647],\n",
      "         [  5.0383, -15.2856,  -6.3635,  ...,  -4.1087,  -2.2531,   3.3804],\n",
      "         [  1.4636,  -6.5796,  -7.8802,  ...,   2.4015,  -4.3172,  -0.6166],\n",
      "         ...,\n",
      "         [  4.2191,  -8.1842,  -6.7188,  ...,  -3.4153, -10.8029,  -0.9748],\n",
      "         [  2.6603,  -6.0252,   1.5291,  ...,  -3.8681, -11.3783,   1.0482],\n",
      "         [  1.1298,  -3.0554,  -5.6794,  ...,  -3.4366,  -0.5796,   3.4665]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ -7.3419,  -7.8615,   1.4584,  ..., -15.4116,  11.3768,   1.3392],\n",
      "         [ -1.5447, -10.3084,  -4.8368,  ...,  -1.5623,  -7.6700,  -1.9189],\n",
      "         [ -4.4537,  -6.7122,  -9.4838,  ...,   2.1586,  -7.3530,  -1.0329],\n",
      "         ...,\n",
      "         [ -4.8644,  -0.3763,  -5.6682,  ...,   2.0764, -14.9502,   5.6812],\n",
      "         [  0.6092,  -2.6496,  -3.9657,  ...,  -0.6118, -13.3693,   8.0960],\n",
      "         [ -2.7073,  -3.0969,  -3.0182,  ...,  -9.9187,   3.3660,   7.1145]],\n",
      "\n",
      "        [[ -4.8137,  -7.2091,   2.5273,  ..., -14.0200,  12.4470,   4.5603],\n",
      "         [  1.8266, -13.9175,  -4.6695,  ...,  -3.8571,  -1.5251,   1.7437],\n",
      "         [ -2.7767,  -3.3307,  -8.6021,  ...,   3.4260,  -5.3577,  -0.0436],\n",
      "         ...,\n",
      "         [ -1.1980,  -3.2996,  -7.0665,  ...,  -3.8936, -12.9943,   4.2693],\n",
      "         [ -1.1258,  -2.1721,   1.3643,  ...,  -3.7653, -12.8803,   5.3227],\n",
      "         [ -3.3197,  -1.7480,  -3.4677,  ...,  -7.0325,  -0.0364,   8.6409]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-12.8261,  -7.4316,  -0.5133,  ..., -17.0109,   4.1412,   0.4096],\n",
      "         [ -6.5062,  -8.4287,  -5.6811,  ...,  -4.2900, -13.3465,  -5.7844],\n",
      "         [ -9.5166,  -4.9175, -11.6386,  ...,  -1.1773, -13.2623,  -5.2870],\n",
      "         ...,\n",
      "         [ -5.2797,   1.4417,  -3.8449,  ...,   2.5516, -21.7542,   3.4538],\n",
      "         [ -1.7103,  -1.1852,  -4.0556,  ...,  -0.4589, -20.1812,   5.8275],\n",
      "         [ -5.1958,  -2.9399,  -1.9038,  ...,  -8.6651,  -2.0524,   7.8081]],\n",
      "\n",
      "        [[-10.8017,  -6.5798,   1.1878,  ..., -16.4577,   5.6943,   3.6485],\n",
      "         [ -4.1766, -12.7859,  -5.2923,  ...,  -6.8078,  -7.1958,  -2.9920],\n",
      "         [ -9.1510,  -0.7565,  -9.3503,  ...,   2.1515, -11.9064,  -4.2890],\n",
      "         ...,\n",
      "         [ -5.2914,  -1.0864,  -5.1300,  ...,  -4.2237, -21.2085,   4.0670],\n",
      "         [ -4.5928,  -1.5694,   1.7656,  ...,  -2.8921, -19.5010,   2.4657],\n",
      "         [ -6.2044,  -1.1009,  -2.9340,  ...,  -5.2437,  -5.5595,   9.3982]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[-1.4177e+01, -1.0818e+01, -2.0517e-02,  ..., -1.3899e+01,\n",
      "          -2.4198e-01,  5.2407e+00],\n",
      "         [-8.2758e+00, -8.7449e+00, -1.0636e+01,  ...,  1.0777e+00,\n",
      "          -1.7978e+01, -3.7788e+00],\n",
      "         [-4.5669e+00, -5.2559e+00, -1.7623e+01,  ...,  1.5141e+00,\n",
      "          -1.5986e+01, -3.8160e+00],\n",
      "         ...,\n",
      "         [-6.3453e+00, -1.9768e+00, -8.9621e+00,  ...,  3.6387e+00,\n",
      "          -2.2666e+01,  3.5626e+00],\n",
      "         [-2.7860e+00, -3.2257e+00, -1.0460e+01,  ...,  1.5412e+00,\n",
      "          -2.3105e+01,  6.2408e+00],\n",
      "         [-7.3711e+00, -7.7884e+00,  9.5593e-01,  ..., -8.2310e+00,\n",
      "          -3.9383e+00,  9.6491e+00]],\n",
      "\n",
      "        [[-1.2117e+01, -1.0498e+01,  1.0643e+00,  ..., -1.3895e+01,\n",
      "           2.0773e+00,  7.9525e+00],\n",
      "         [-5.6647e+00, -1.3375e+01, -9.0223e+00,  ..., -2.6933e+00,\n",
      "          -1.0556e+01, -1.8339e+00],\n",
      "         [-1.0293e+01,  9.8391e-02, -1.6372e+01,  ...,  4.7898e+00,\n",
      "          -1.5176e+01, -4.4241e+00],\n",
      "         ...,\n",
      "         [-5.8969e+00, -3.5736e+00, -1.1650e+01,  ..., -4.6095e+00,\n",
      "          -2.4662e+01,  7.0034e+00],\n",
      "         [-5.8335e+00, -2.6239e+00, -3.2992e+00,  ..., -2.2592e+00,\n",
      "          -2.2638e+01,  4.1905e+00],\n",
      "         [-8.2542e+00, -4.9109e+00, -7.5076e-01,  ..., -4.5657e+00,\n",
      "          -8.8791e+00,  1.0962e+01]]], grad_fn=<AddBackward0>), tensor([[[-18.7943,   0.0985,  -6.6211,  ..., -16.4377,  -4.9953,   1.1963],\n",
      "         [-12.8383,   1.4108, -19.5417,  ...,  -1.2756, -21.7121,  -4.5869],\n",
      "         [ -8.8693,   4.7238, -27.1704,  ...,   0.2982, -18.7135,  -4.4651],\n",
      "         ...,\n",
      "         [-11.2362,  10.0784, -15.8307,  ...,   5.0667, -28.8834,   2.5526],\n",
      "         [ -5.9691,   9.8257, -18.3235,  ...,   1.5493, -29.0726,   6.4366],\n",
      "         [ -7.6355,   7.6140,  -8.8891,  ...,  -9.6098,  -9.5481,  10.7948]],\n",
      "\n",
      "        [[-16.8304,   1.1778,  -7.1839,  ..., -18.1786,  -4.0175,   2.6859],\n",
      "         [ -9.5452,  -1.6047, -16.6443,  ...,  -5.0691, -16.0754,  -3.9018],\n",
      "         [-15.7408,  10.7822, -23.5130,  ...,   2.6446, -19.7654,  -4.5719],\n",
      "         ...,\n",
      "         [ -6.2152,  13.3432, -18.5333,  ...,  -2.6764, -28.5261,  11.2181],\n",
      "         [ -8.8618,  11.5948, -10.6256,  ...,  -3.5785, -26.7655,   7.3544],\n",
      "         [ -8.4863,   9.9941, -10.4587,  ...,  -5.5475, -15.8909,  12.7997]]],\n",
      "       grad_fn=<AddBackward0>), tensor([[[ 1.3384e-01, -6.2243e+00, -1.3022e+01,  ...,  1.3729e+01,\n",
      "          -2.4797e+00, -8.7180e+00],\n",
      "         [ 3.0209e+00, -2.7901e+00, -3.8091e+01,  ...,  2.9943e+01,\n",
      "          -1.8076e+01, -4.0236e+00],\n",
      "         [ 6.8729e+00,  2.6780e-01, -4.6731e+01,  ...,  2.8560e+01,\n",
      "          -1.6809e+01, -7.0668e+00],\n",
      "         ...,\n",
      "         [ 5.6747e+00,  2.7156e+00, -3.5233e+01,  ...,  2.8486e+01,\n",
      "          -3.1411e+01,  4.4234e+00],\n",
      "         [ 1.1834e+01, -1.5887e-02, -3.6577e+01,  ...,  3.0541e+01,\n",
      "          -3.2460e+01,  7.9123e+00],\n",
      "         [ 8.7265e+00, -1.0999e+00, -1.5026e+01,  ...,  2.6505e+01,\n",
      "          -1.2920e+01,  2.6611e+00]],\n",
      "\n",
      "        [[-1.6558e+00, -8.9843e+00, -1.7743e+01,  ...,  1.5737e+01,\n",
      "           9.6979e-01, -3.1070e+00],\n",
      "         [ 6.3268e+00, -7.8145e+00, -3.2088e+01,  ...,  2.4954e+01,\n",
      "          -1.3842e+01, -3.4138e+00],\n",
      "         [ 5.9959e+00,  2.1324e+00, -4.4882e+01,  ...,  3.1262e+01,\n",
      "          -2.2614e+01, -6.2475e+00],\n",
      "         ...,\n",
      "         [ 1.5184e+01,  1.3672e-02, -3.6547e+01,  ...,  2.6233e+01,\n",
      "          -3.5108e+01,  1.8368e+01],\n",
      "         [ 9.5068e+00,  3.9829e+00, -2.7143e+01,  ...,  2.4713e+01,\n",
      "          -3.3582e+01,  1.3144e+01],\n",
      "         [ 3.3789e+00,  5.7193e+00, -1.8026e+01,  ...,  2.9281e+01,\n",
      "          -2.2592e+01,  4.7912e+00]]], grad_fn=<AddBackward0>), tensor([[[ -42.1912,  -16.9952,  -95.3213,  ...,    8.3718,  -11.7452,\n",
      "            34.3229],\n",
      "         [ -37.4156,  -10.6368, -119.4785,  ...,   16.1470,  -19.6849,\n",
      "            27.4863],\n",
      "         [ -39.8820,  -11.7989, -124.5471,  ...,   14.0733,  -17.1801,\n",
      "            26.1404],\n",
      "         ...,\n",
      "         [ -32.6469,  -16.3285, -116.9647,  ...,   15.2599,  -31.0337,\n",
      "            22.0916],\n",
      "         [ -27.9409,  -13.7538, -119.0643,  ...,   15.4582,  -28.1907,\n",
      "            29.7226],\n",
      "         [ -20.5192,   -4.8161,  -99.4872,  ...,   16.5048,  -21.4422,\n",
      "            28.4232]],\n",
      "\n",
      "        [[ -40.2203,  -18.2700,  -99.2928,  ...,    7.0219,   -5.1307,\n",
      "            43.4452],\n",
      "         [ -27.5474,  -16.6919, -117.1980,  ...,   10.8627,   -6.7921,\n",
      "            29.5967],\n",
      "         [ -30.3638,   -9.8063, -132.0924,  ...,   21.7393,  -15.7703,\n",
      "            27.1751],\n",
      "         ...,\n",
      "         [ -10.3464,  -12.0951, -121.5368,  ...,    7.8199,  -43.5899,\n",
      "            36.5394],\n",
      "         [ -17.6759,   -9.8230, -110.8841,  ...,    7.0945,  -38.6185,\n",
      "            31.2148],\n",
      "         [ -13.2246,    8.4539, -101.4101,  ...,   22.8037,  -32.0585,\n",
      "            30.9473]]], grad_fn=<AddBackward0>), tensor([[[-0.3784, -0.0016, -0.4596,  ..., -0.1236, -0.1294, -0.1358],\n",
      "         [-0.3657,  0.0544, -0.5358,  ..., -0.1036, -0.1933, -0.1362],\n",
      "         [-0.3564,  0.0672, -0.4977,  ..., -0.1213, -0.1833, -0.0985],\n",
      "         ...,\n",
      "         [-0.3688,  0.0134, -0.4067,  ...,  0.0032, -0.2446, -0.1936],\n",
      "         [-0.3828,  0.0139, -0.3928,  ..., -0.0427, -0.2463, -0.1160],\n",
      "         [-0.3299,  0.0468, -0.3189,  ..., -0.0420, -0.2244, -0.1322]],\n",
      "\n",
      "        [[-0.3626,  0.0260, -0.4058,  ..., -0.1254, -0.1057, -0.0410],\n",
      "         [-0.3140,  0.0339, -0.4643,  ..., -0.1246, -0.1213, -0.1267],\n",
      "         [-0.3249,  0.0698, -0.5137,  ..., -0.0857, -0.1562, -0.1067],\n",
      "         ...,\n",
      "         [-0.2675,  0.0508, -0.4767,  ..., -0.0773, -0.2989, -0.1043],\n",
      "         [-0.3510,  0.0958, -0.4259,  ..., -0.0913, -0.3067, -0.1233],\n",
      "         [-0.3109,  0.1279, -0.3786,  ..., -0.0266, -0.2726, -0.1186]]],\n",
      "       grad_fn=<NativeLayerNormBackward0>)), attentions=None)\n"
     ]
    }
   ],
   "source": [
    "output = model(audio,output_hidden_states= True)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 999, 1024])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.hidden_states[5].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### detokenizer\n",
    "from sklearn.cluster import KMeans as KMeans\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 3, 4)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kMeans = KMeans(2)\n",
    "x = np.random.rand(2,3,4)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KMeansBatch:\n",
    "    def __init__(self, num_cluster):\n",
    "        self.model = KMeans(num_cluster, n_init=\"auto\")\n",
    "        self.result = None\n",
    "    \n",
    "    def fit(self, x:torch.Tensor):\n",
    "        \"\"\"\n",
    "        x : [B, T, F]\n",
    "        return [B,T,1] utilizing kmeans\n",
    "        \"\"\"\n",
    "        input = x.cpu().numpy()\n",
    "        res = []\n",
    "        for element in x:\n",
    "            self.model.fit(element)\n",
    "            labels = self.model.labels_\n",
    "            res.append(labels)\n",
    "        res = torch.from_numpy(np.array(res))\n",
    "        return res\n",
    "\n",
    "    def getResult(self):\n",
    "        return self.result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-1.9510,  0.0655, -0.1697,  ..., -1.8086,  1.3107, -0.9404],\n",
       "         [-0.0846,  0.3714,  0.6525,  ...,  0.8467,  0.3801,  0.1638],\n",
       "         [-0.5826,  1.3534,  1.1715,  ..., -0.3452,  0.7957,  0.4857],\n",
       "         ...,\n",
       "         [-1.5848, -0.6641, -1.4767,  ..., -0.3453,  0.7297, -0.4430],\n",
       "         [ 0.0998, -1.6627,  0.9971,  ...,  0.7772,  0.2793, -1.3058],\n",
       "         [-0.9017, -1.1755, -0.3186,  ..., -0.0060,  0.8930,  0.3069]],\n",
       "\n",
       "        [[ 0.4218, -1.3497, -0.4939,  ..., -2.3036,  0.7555, -0.4534],\n",
       "         [ 0.5663, -0.8404,  0.0966,  ...,  1.6001, -0.4544, -1.4242],\n",
       "         [-0.1440,  1.5499,  1.1053,  ...,  0.6999, -0.7755, -1.5344],\n",
       "         ...,\n",
       "         [-1.5627, -0.2969, -0.7342,  ..., -0.4407, -0.0243, -0.7323],\n",
       "         [-0.4744,  0.0621,  0.7322,  ..., -1.1942, -0.8345, -0.4831],\n",
       "         [-0.5980, -0.4319, -0.3590,  ...,  0.9624,  0.2583, -0.7888]],\n",
       "\n",
       "        [[ 0.8524,  0.0817, -0.3299,  ...,  1.0080,  0.9640, -0.9066],\n",
       "         [ 1.0203, -0.5707,  1.5764,  ..., -0.7557,  1.3650, -0.2793],\n",
       "         [-1.0828, -0.3853, -0.9090,  ...,  2.0081,  0.4665,  0.3329],\n",
       "         ...,\n",
       "         [ 1.2904, -0.2034,  1.4591,  ..., -1.2870,  0.0125, -1.7208],\n",
       "         [-2.0922,  0.0421,  0.5063,  ...,  0.2624, -0.2324, -0.8402],\n",
       "         [ 0.4550,  0.7769,  2.6789,  ..., -0.0376,  0.6231, -2.7282]],\n",
       "\n",
       "        ...,\n",
       "\n",
       "        [[-0.6949, -0.2935, -0.2366,  ..., -1.1438,  0.8403,  0.4353],\n",
       "         [-1.5313, -2.3276,  1.1558,  ..., -1.3219,  0.1769,  0.8301],\n",
       "         [-2.6065,  0.2649,  1.8820,  ..., -1.4175, -1.0546,  1.0638],\n",
       "         ...,\n",
       "         [ 0.2966,  0.9789, -0.8192,  ...,  0.8245, -0.6932, -0.8942],\n",
       "         [-0.0280,  0.3463, -0.3005,  ...,  0.3576, -0.6343,  1.0084],\n",
       "         [ 1.0740,  1.6433, -0.8546,  ..., -0.4162, -0.8596, -0.3674]],\n",
       "\n",
       "        [[-1.1094, -1.2346, -0.5110,  ...,  0.1860, -0.3202, -0.8605],\n",
       "         [-2.0107,  1.4500,  0.4645,  ..., -0.8892, -0.9798,  0.2017],\n",
       "         [ 0.7165, -0.8105,  1.7214,  ..., -0.8152, -0.9525,  0.1026],\n",
       "         ...,\n",
       "         [ 1.4189, -0.6122,  0.4128,  ...,  1.2370, -0.4966,  0.9491],\n",
       "         [ 0.6996, -1.4449, -0.1834,  ...,  2.2283, -0.4338,  0.6118],\n",
       "         [ 0.0779, -1.3105, -1.3245,  ...,  2.2079,  0.7334, -1.1683]],\n",
       "\n",
       "        [[ 0.3368, -0.9671,  0.8519,  ..., -0.7028,  0.7287,  0.2486],\n",
       "         [ 0.4009,  0.6668,  0.1689,  ...,  0.7489, -0.2233,  0.3035],\n",
       "         [-1.1692,  0.2512,  0.5720,  ...,  0.5523,  0.3510,  0.2636],\n",
       "         ...,\n",
       "         [-1.2216, -0.0785,  1.2978,  ..., -0.2854, -0.7471, -0.6386],\n",
       "         [ 1.0421, -1.2651, -1.2289,  ...,  0.2092, -0.5958,  2.2313],\n",
       "         [-0.2773, -0.2529, -1.4425,  ...,  1.0241,  0.1694, -0.6228]]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kMeansBatch = KMeansBatch(300)\n",
    "x = torch.randn(32,512)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n",
      "c:\\Users\\Beilong Tang\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "output = kMeansBatch.fit(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 30])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array([0, 0, 1]), array([1, 0, 0])]\n"
     ]
    }
   ],
   "source": [
    "print(output)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
